{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "88716015",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6dc057d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://huggingface.co/huseinzol05/bpe/resolve/main/ms-en.subwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ea89d350",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-04 22:02:50.018541: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
      "/home/ubuntu/tf-nvidia/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from malaya.text.t2t import text_encoder\n",
    "import malaya"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "73a93be0",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = text_encoder.SubwordTextEncoder('ms-en.subwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3b11e900",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.decode([25891])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "87943d78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[197]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.encode('saya')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6bde4831",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25880"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "49a59f3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !~/tf-nvidia/bin/pip3 install fasttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "80e9cf0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
     ]
    }
   ],
   "source": [
    "fast_text = malaya.language_detection.fasttext()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "38c3e1ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['malay', 'eng']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fast_text.predict(['saya suka', 'i like'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d7bc9727",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder:\n",
    "    def __init__(self, encoder):\n",
    "        self.encoder = encoder\n",
    "        self.vocab_size = encoder.vocab_size\n",
    "\n",
    "    def encode(self, s):\n",
    "        s = [self.encoder.encode(s_) for s_ in s]\n",
    "        s = [i + [1] for i in s]\n",
    "        return s\n",
    "\n",
    "    def decode(self, ids, strip_extraneous = False):\n",
    "        return self.encoder.decode(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9ba85517",
   "metadata": {},
   "outputs": [],
   "source": [
    "s_encoder = Encoder(encoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "629e6e19",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.data_generators import problem\n",
    "from tensor2tensor.data_generators import text_problems\n",
    "from tensor2tensor.utils import registry\n",
    "from tqdm import tqdm\n",
    "from glob import glob\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d66abf1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['augmented-ms-en-v2.json',\n",
       " 'augmented-ms-en-3.json',\n",
       " 'augmented-ms-en-2.json',\n",
       " 'augmented-ms-en-test.json',\n",
       " 'augmented-ms-en-v3.json',\n",
       " 'augmented-ms-en-1.json']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "augmented = glob('augmented-ms-en-*.json')\n",
    "augmented"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "70526163",
   "metadata": {},
   "outputs": [],
   "source": [
    "@registry.register_problem\n",
    "class Translation(text_problems.Text2TextProblem):\n",
    "    @property\n",
    "    def approx_vocab_size(self):\n",
    "        return encoder.vocab_size\n",
    "\n",
    "    @property\n",
    "    def is_generate_per_split(self):\n",
    "        # generate_data will shard the data into TRAIN and EVAL for us.\n",
    "        return False\n",
    "\n",
    "    @property\n",
    "    def dataset_splits(self):\n",
    "        return [\n",
    "            {'split': problem.DatasetSplit.TRAIN, 'shards': 100},\n",
    "        ]\n",
    "\n",
    "    def generate_samples(self, data_dir, tmp_dir, dataset_split):\n",
    "\n",
    "        for file in augmented:\n",
    "            with open(file) as fopen:\n",
    "                data = json.load(fopen)\n",
    "\n",
    "            for i in tqdm(range(len(data['ms']))):\n",
    "                if len(data['ms'][i]) and len(data['en'][i]):\n",
    "                    i, o = s_encoder.encode([data['ms'][i], data['en'][i]])\n",
    "                    yield {'inputs': i, 'targets': o}\n",
    "                \n",
    "        with open('train/left.txt') as fopen:\n",
    "            left = fopen.read().split('\\n')\n",
    "        \n",
    "        with open('train/right.txt') as fopen:\n",
    "            right = fopen.read().split('\\n')\n",
    "            \n",
    "        \n",
    "        for i in tqdm(range(len(left))):\n",
    "            if len(left[i]) and len(right[i]):\n",
    "                lang_left, lang_right = fast_text.predict([left[i], right[i]])\n",
    "                if lang_left not in ['malay', 'eng']:\n",
    "                    continue\n",
    "                if lang_right not in ['eng']:\n",
    "                    continue\n",
    "                i, o = s_encoder.encode([left[i], right[i]])\n",
    "                yield {'inputs': i, 'targets': o}\n",
    "\n",
    "    def generate_encoded_samples(self, data_dir, tmp_dir, dataset_split):\n",
    "\n",
    "        generator = self.generate_samples(data_dir, tmp_dir, dataset_split)\n",
    "        for sample in generator:\n",
    "            yield sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c74aa440",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "160edb4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.system('rm -rf t2t-noisy-ms-en/data')\n",
    "DATA_DIR = os.path.expanduser('t2t-noisy-ms-en/data')\n",
    "TMP_DIR = os.path.expanduser('t2t-noisy-ms-en/tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a420c0d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_26077/2493044156.py:1: The name tf.gfile.MakeDirs is deprecated. Please use tf.io.gfile.makedirs instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.gfile.MakeDirs(DATA_DIR)\n",
    "tf.gfile.MakeDirs(TMP_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "deaf62c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.utils import registry\n",
    "from tensor2tensor import problems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e02f52a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|                                                                                                                       | 0/84178 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 0.\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 84178/84178 [00:09<00:00, 9268.54it/s]\n",
      "  5%|████▋                                                                                                   | 15131/335872 [00:01<00:32, 9750.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 100000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 100000.\n",
      " 34%|██████████████████████████████████▉                                                                   | 114886/335872 [00:11<00:21, 10517.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 200000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 200000.\n",
      " 64%|█████████████████████████████████████████████████████████████████▍                                    | 215451/335872 [00:21<00:11, 10562.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 300000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 300000.\n",
      " 94%|███████████████████████████████████████████████████████████████████████████████████████████████▊      | 315557/335872 [00:31<00:01, 10187.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 400000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 400000.\n",
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 335872/335872 [00:33<00:00, 9963.50it/s]\n",
      " 58%|████████████████████████████████████████████████████████████▏                                           | 79919/138131 [00:07<00:06, 9413.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 500000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 500000.\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 138131/138131 [00:13<00:00, 10157.06it/s]\n",
      " 63%|█████████████████████████████████████████████████████████████████▉                                       | 41243/65642 [00:04<00:02, 9648.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 600000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 600000.\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 65642/65642 [00:06<00:00, 10187.39it/s]\n",
      " 18%|██████████████████▊                                                                                     | 76119/419750 [00:08<00:34, 9996.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 700000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 700000.\n",
      " 42%|██████████████████████████████████████████▉                                                            | 175234/419750 [00:18<00:25, 9564.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 800000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 800000.\n",
      " 66%|███████████████████████████████████████████████████████████████████                                   | 275785/419750 [00:28<00:13, 10301.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 900000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 900000.\n",
      " 89%|████████████████████████████████████████████████████████████████████████████████████████████▏          | 375503/419750 [00:39<00:04, 9892.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1000000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1000000.\n",
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████| 419750/419750 [00:43<00:00, 9606.69it/s]\n",
      " 21%|█████████████████████▏                                                                                 | 56116/273070 [00:05<00:21, 10222.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1100000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1100000.\n",
      " 57%|██████████████████████████████████████████████████████████▏                                           | 155693/273070 [00:15<00:10, 11090.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1200000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1200000.\n",
      " 94%|███████████████████████████████████████████████████████████████████████████████████████████████▌      | 255830/273070 [00:25<00:01, 10722.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1300000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1300000.\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 273070/273070 [00:26<00:00, 10213.82it/s]\n",
      "  3%|██▉                                                                                                   | 104645/3712555 [00:21<12:03, 4985.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1400000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1400000.\n",
      "  6%|██████▎                                                                                               | 230534/3712555 [00:45<11:08, 5212.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1500000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1500000.\n",
      " 10%|█████████▊                                                                                            | 356284/3712555 [01:09<10:31, 5316.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1600000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1600000.\n",
      " 13%|█████████████▏                                                                                        | 481573/3712555 [01:35<10:47, 4986.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1700000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1700000.\n",
      " 16%|████████████████▋                                                                                     | 606974/3712555 [02:02<11:08, 4644.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1800000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1800000.\n",
      " 20%|████████████████████                                                                                  | 732475/3712555 [02:27<09:18, 5340.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1900000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 1900000.\n",
      " 23%|███████████████████████▌                                                                              | 857928/3712555 [02:51<09:01, 5272.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2000000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2000000.\n",
      " 26%|███████████████████████████                                                                           | 983710/3712555 [03:16<09:12, 4938.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2100000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2100000.\n",
      " 30%|██████████████████████████████▏                                                                      | 1109725/3712555 [03:40<08:19, 5207.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2200000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2200000.\n",
      " 33%|█████████████████████████████████▌                                                                   | 1235111/3712555 [04:05<08:37, 4787.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2300000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2300000.\n",
      " 37%|█████████████████████████████████████                                                                | 1361101/3712555 [04:29<07:11, 5448.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2400000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2400000.\n",
      " 40%|████████████████████████████████████████▍                                                            | 1486406/3712555 [04:54<07:02, 5274.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2500000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2500000.\n",
      " 43%|███████████████████████████████████████████▊                                                         | 1612648/3712555 [05:18<06:40, 5241.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2600000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2600000.\n",
      " 47%|███████████████████████████████████████████████▎                                                     | 1738168/3712555 [05:43<06:06, 5381.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2700000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2700000.\n",
      " 50%|██████████████████████████████████████████████████▋                                                  | 1863968/3712555 [06:08<06:11, 4978.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2800000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2800000.\n",
      " 54%|██████████████████████████████████████████████████████▏                                              | 1990053/3712555 [06:33<05:49, 4927.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2900000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 2900000.\n",
      " 57%|█████████████████████████████████████████████████████████▌                                           | 2115361/3712555 [06:58<05:01, 5304.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3000000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3000000.\n",
      " 60%|████████████████████████████████████████████████████████████▉                                        | 2241628/3712555 [07:23<04:45, 5144.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3100000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3100000.\n",
      " 64%|████████████████████████████████████████████████████████████████▍                                    | 2367230/3712555 [07:47<04:14, 5290.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3200000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3200000.\n",
      " 67%|███████████████████████████████████████████████████████████████████▊                                 | 2492897/3712555 [08:12<03:47, 5350.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3300000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3300000.\n",
      " 71%|███████████████████████████████████████████████████████████████████████▏                             | 2618326/3712555 [08:37<03:30, 5198.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3400000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3400000.\n",
      " 74%|██████████████████████████████████████████████████████████████████████████▋                          | 2743927/3712555 [09:02<03:20, 4830.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3500000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3500000.\n",
      " 77%|██████████████████████████████████████████████████████████████████████████████                       | 2869527/3712555 [09:27<02:35, 5412.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3600000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3600000.\n",
      " 81%|█████████████████████████████████████████████████████████████████████████████████▍                   | 2995066/3712555 [09:52<02:15, 5305.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3700000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3700000.\n",
      " 84%|████████████████████████████████████████████████████████████████████████████████████▉                | 3121213/3712555 [10:17<01:49, 5388.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3800000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3800000.\n",
      " 87%|████████████████████████████████████████████████████████████████████████████████████████▎            | 3246257/3712555 [10:41<01:28, 5252.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3900000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 3900000.\n",
      " 91%|███████████████████████████████████████████████████████████████████████████████████████████▋         | 3372203/3712555 [11:06<01:07, 5050.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 4000000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 4000000.\n",
      " 94%|███████████████████████████████████████████████████████████████████████████████████████████████▏     | 3497539/3712555 [11:31<00:44, 4798.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 4100000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 4100000.\n",
      " 98%|██████████████████████████████████████████████████████████████████████████████████████████████████▌  | 3623299/3712555 [11:55<00:22, 3952.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 4200000.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generating case 4200000.\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████| 3712555/3712555 [12:13<00:00, 5062.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generated 4270649 Examples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Generated 4270649 Examples\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Shuffling data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Shuffling data...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Data shuffled.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Data shuffled.\n"
     ]
    }
   ],
   "source": [
    "PROBLEM = 'translation'\n",
    "t2t_problem = problems.problem(PROBLEM)\n",
    "t2t_problem.generate_data(DATA_DIR, TMP_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c9d070b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf1",
   "language": "python",
   "name": "tf1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
